#!/usr/bin/env coffee

import readline from 'readline'
import fetch from 'node-fetch'
import fastcsv from 'fast-csv'
import fs from 'fs'
import {resolve} from 'path'
import {thisdir,thisfile} from '@rmw/thisfile'
import {open} from 'lmdb-store'
import ss from 'stream-string'

{format} = fastcsv

DIR = thisdir `import.meta`
DB = open({
  path:DIR+"/dump.db"
  encoding:'binary'
  compression:true
})

dump = (keyword)=>
  if DB.get(keyword)
    return

  r = await fetch("https://www.taosj.com/data/industry/category/search/list?name=#{encodeURIComponent(keyword)}", {
    "headers": {
      "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
      "accept-language": "zh-CN,zh;q=0.9,zh-TW;q=0.8,zh-HK;q=0.7,en;q=0.6",
      "cache-control": "max-age=0",
      "sec-fetch-dest": "document",
      "sec-fetch-mode": "navigate",
      "sec-fetch-site": "none",
      "sec-fetch-user": "?1",
      "upgrade-insecure-requests": "1",
      "cookie": "business_type=; MEIQIA_TRACK_ID=1cT843DsF1gDOtR60ueoJPlxhcG; Hm_lvt_cdfb35f92929b99ae60e80a03194be56=1605858676; _ga=GA1.2.1368924654.1605858676; _gid=GA1.2.648977350.1605858676; t_auth=76585e0a5eccb33cb3310752d3f188f1; auth=52e263359ae1e65c6e66b3bb5a36a21e96a55420; token=52e263359ae1e65c6e66b3bb5a36a21e96a55420; event=login; only_mobile=13338067701; user=806768; mobile=13338067701; nick=%E9%B9%8F; id=806768; signup_time=2020-05-27%2008%3A59%3A37; kj_version=%E8%B7%A8%E5%A2%83%E7%94%B5%E5%95%86%E9%9D%92%E9%93%9C%E4%BC%9A%E5%91%98; kj_expire_date=; kj_status=%E6%9C%89%E6%95%88; kj_order_date=; tb_version=%E8%AF%95%E7%94%A8%E7%89%88; tb_status=%E6%9C%89%E6%95%88; tb_expire_date=; pdd_version=%E5%85%8D%E8%B4%B9%E7%89%88; pdd_status=%E6%9C%89%E6%95%88; pdd_expire_date=; all_version=0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0; MEIQIA_VISIT_ID=1kXtKp24YxwVQGWXslPMIBFMSP9; Hm_lpvt_cdfb35f92929b99ae60e80a03194be56=1605858707; is_new=1; AGL_USER_ID=5afee2fd-f00b-47f2-b83b-9dec8ab0aef0; JSESSIONID=6FE190F3BD2F33A397BE462CB1616AD4"
    },
    "referrerPolicy": "strict-origin-when-cross-origin",
    "body": null,
    "method": "GET",
    "mode": "cors"
  })
  {data} = await r.json()
  stream = format({})
  p = ss(stream)
  for i from data.result
    for j from i.catList
      stream.write [j.parentCid, j.cid, j.name]
  stream.end()
  txt = await p
  await DB.put(keyword, txt)



do =>
  rl = readline.createInterface({
      input: fs.createReadStream(DIR+'/key.txt'),
      # output: process.stdout
      console: false
      crlfDelay: Infinity
  })

  n=0
  for await keyword from rl
    console.log ++n, keyword
    await dump keyword
  s = fs.createWriteStream DIR+"/kind.csv"
  s.write '\ufeff'+'父类型ID,类型ID,类型名,关键词\n'
  for {key,value} from DB.getRange()
    s.write ",,,"+key+"\n"
    s.write value.toString('utf8')+"\n"

  s.on 'finish',process.exit
  s.end()
